package com.spider.silence.downloader;

import java.io.File;
import java.lang.reflect.InvocationTargetException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.concurrent.*;

import com.spider.silence.utils.*;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.dom4j.Element;
import org.jsoup.Jsoup;
import org.jsoup.select.Elements;
import org.openqa.selenium.Cookie;

import us.codecraft.webmagic.Page;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.task.BaiduForumTask;
import com.spider.silence.task.manager.CommonTaskManager;

/**
 * Created by 赵欢 on 2017/1/6.
 * 百度贴吧下载器
 */
public class BaiduForumDownloader extends AbstractCommonDownLoader implements Cloneable {

    protected List<HashMap<String, Object>> forumsList = new ArrayList<HashMap<String, Object>>();

    private static Logger logger = Logger.getLogger(BaiduForumDownloader.class);

    private static int default_pageNum = 50;

    private Map<String, File> files = null;

    public Map<String, File> getFiles() {
        return files;
    }

    public BaiduForumDownloader() {

    }

    public BaiduForumDownloader initForums() {
        org.dom4j.Document document = XmlUtil.getConf("META-INF/BaiduForum.xml");
        if (document != null) {
            Element elementRoot = document.getRootElement();
            for (Iterator it = elementRoot.elementIterator(); it.hasNext(); ) {
                Element element = (Element) it.next();
                HashMap<String, Object> forumMap = new HashMap<String, Object>();
                forumMap.put("url", element.element("forum-url").getText());
                forumMap.put("name", element.element("forum-name").getText());
                forumMap.put("pageNum", element.element("forum_pageNum").getText());
                forumMap.put("type", element.element("type").getText());
                forumsList.add(forumMap);
            }
        }
        return this;
    }


    /**
     * 添加任务
     */
    public void generateTask() {
        new Thread(new Runnable() {
            @Override
            public void run() {
                if (forumsList != null) {
                    for (HashMap<String, Object> map : forumsList) {
                        int page = Integer.valueOf(map.get("pageNum").toString()) / default_pageNum;
                        //获取分页的document
                        for (int i = 0; i < page; i++) {
                            String url = map.get("url").toString() + "" + i * default_pageNum;
                            try {
                                Page pageContent = getHtml(url, true);
                                long startTime = System.currentTimeMillis();
                                long nowTime = startTime;
                                while (null == pageContent && nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("BaiduForum.timeOut"))) {
                                    pageContent = getHtml(url, true);
                                    nowTime = System.currentTimeMillis();
                                }
                                if(null == pageContent) {
                                    continue;
                                }
                                if(pageContent.getStatusCode() == 403) {
                                    continue;
                                }
                                if (null != pageContent.getHtml()) {
                                    Document document = pageContent.getHtml().getDocument();
                                    String html = document.html();
                                    //html 内容都在注释内，需要干掉注销符号
                                    html = html.replaceAll("<!--", " ");
                                    html = html.replaceAll("-->", " ");
                                    Elements elements = Jsoup.parse(html).select(".j_thread_list");
                                    for (org.jsoup.nodes.Element element : elements) {
                                        //map.put("element",element);
                                        JSONObject jsonObject = JSONObject.parseObject(element.attr("data-field"));
                                        Object is_top = jsonObject.get("is_top");
                                        if (is_top == null) {
                                            BaiduForumTask task = new BaiduForumTask();
                                            String forum_url = element.select(".threadlist_title").select("a").attr("href");
                                            forum_url = "http://tieba.baidu.com" + forum_url;
                                            String tId = jsonObject.get("id").toString();
                                            task.setUrl(forum_url);
                                            task.settId(tId);
                                            task.setType(map.get("type").toString());
                                            task.setForumName(map.get("name").toString());
                                            try {
                                                CommonTaskManager.add("baiduForumTask", task);
                                            } catch (InterruptedException e) {
                                                //e.printStackTrace();
                                                logger.error(e);
                                            }
                                        }

                                    }
                                }
                            } catch (Exception e) {
                                System.err.println("生成爬虫任务失败");
                                continue;
                            }
                        }
                        //pool.shutdown();
                    }
                }
            }
        }).start();
    }

    /**
     * 爬取各个帖子
     *
     * @param
     * @return
     */
    public JSONObject crawlItem(String url, String tid, String type) throws Exception {
        JSONObject item = new JSONObject();
        JSONObject data = new JSONObject();
        boolean success = false;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        while (nowTime - startTime < Long.valueOf(PropertiesUtil.getVal("BaiduForum.timeOut"))) {
            data = new JSONObject();
            data.put("url", url);
            files = new ConcurrentHashMap<>();
            nowTime = System.currentTimeMillis();
            Page page = null;
            try {
                page = getHtml(url, true);
                if(null != page && page.getStatusCode() == 200 && null != page.getHtml().getDocument()) {
                    Document document = page.getHtml().getDocument();
                    String publish_user = document.select("#j_p_postlist > div.l_post.j_l_post.l_post_bright.noborder > div.d_author > ul > li.d_name > a").text();
                    String reply_count = document.select("#thread_theme_5 > div.l_thread_info > ul > li:nth-child(2) > span:nth-child(1)").text();
                    String title = document.select("j_core_title_wrap > div.core_title.core_title_theme_bright > h1").text();
                    data.put("url", url);
                    data.put("id", tid);
                    data.put("publish_user", publish_user);
                    data.put("reply_count", reply_count);
                    data.put("title", title);
                    //获取回复页数
                    int replayPageCount = Integer.valueOf(document.select("#thread_theme_5 > div.l_thread_info > ul > li:nth-child(2) > span:nth-child(2)").text());
                    Elements elements = document.select(".l_post");
                    int elmentSize = elements.size();
                    if (elements != null && elmentSize >= 0) {
                        //获取贴子内容--第一条内容
                        JSONObject first_floor = JSONObject.parseObject(elements.get(0).attr("data-field"));
                        JSONObject content = (JSONObject) JSONObject.parse(first_floor.get("content").toString());
                        //用户ID
                        JSONObject userIdObject = JSONObject.parseObject(elements.get(0).select(".d_author> ul > li.d_name").attr("data-field"));
                        if (userIdObject.get("user_id") != null) {
                            data.put("publish_user_id", userIdObject.get("user_id").toString());
                        }
                        String publish_time = "";
                        Elements imgElements = null;
                        JSONArray img_urls = new JSONArray();
                        if (type.equals("0")) {
                            //发布时间
                            publish_time = content.get("date") == null ? "" : content.get("date").toString();
                            //内容
                            data.put("content", FileUtils.filterImg(elements.get(0).select(".p_content").html()));
                            imgElements = elements.get(0).select(".p_content").select("img.BDE_Image");
                            if (imgElements.size() > 0) {
                                for (int imgIndex = 0; imgIndex < imgElements.size(); imgIndex++) {
                                    org.jsoup.nodes.Element imgElement = imgElements.get(imgIndex);
                                    String img_url = imgElement.absUrl("src");
                                    img_urls.add(img_url.replaceAll("\r|\n", ""));
                                }
                            }

                        } else {
                            //发布时间
                            String tail_info = elements.get(0).select(".tail-info").text();
                            publish_time = getTime(tail_info);
                            //内容
                            data.put("content", FileUtils.filterImg(content.get("content") == null ? "" : content.get("content").toString()));
                            List<String> imgList = FileUtils.filterImgSrc(content.get("content") == null ? "" : content.get("content").toString());
                            if (imgList != null && imgList.size() > 0) {
                                for (String img : imgList) {
                                    img_urls.add(img.replaceAll("\r|\n", ""));
                                }
                            }
                        }

                        data.put("img_urls", img_urls);//帖子中的图片
                        data.put("publish_time", publish_time);
                        //发布人头像
                        String publish_user_photo = elements.get(0).select(".p_author_face ").select("img").attr("src");
                        data.put("publish_user_photo", publish_user_photo.replaceAll("\r|\n", ""));
                        //爬取时间
                        SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                        data.put("spider_time", sdf.format(new Date()));

                        long tsp = DateUtils.dateToTimestamp(DateUtils.parseDate(publish_time)).getTime();
                        String filePrefix = "BaiduForum";//文件前缀
                        String fileName = filePrefix + "_" + tsp + "_" + userIdObject.get("user_id").toString() + "_" + tid;
                        String publishDate = DateUtils.formatDate(DateUtils.parseDate(publish_time)).replace("-", "-");
                        String speechesId = "BaiduForum_speeches_" + publishDate + "_" + fileName;
                        File file = FileUtils.createTempFile(speechesId + "_pageNo_1", document.html().toString());
                        files.put(speechesId + "_pageNo_1", file);
                        //帖子首页的文件
                        //获取本页面的回复
                        JSONArray reply_nodes = new JSONArray();
                        //System.out.println("elements:   " +   elements.size());
                        for (int i = 1; i < elements.size(); i++) {
                            try {
                                JSONObject replay_data = new JSONObject();
                                JSONObject replay_floor = JSONObject.parseObject(elements.get(i).attr("data-field"));
                                if (replay_floor == null) {
                                    //干掉广告
                                    continue;
                                }
                                JSONObject replay_content = (JSONObject) JSONObject.parse(replay_floor.get("content").toString());
                                if (replay_floor == null) {
                                    continue;
                                } else {
                                    //回复人
                                    JSONObject replay_author = JSONObject.parseObject(replay_floor.get("author").toString());
                                    replay_data.put("publish_user", replay_author.get("user_name"));
                                    String replyUserId = queryUserId(replay_data.getString("publish_user"));
                                    if(replyUserId != null) {
                                        replay_data.put("publish_user_id", replyUserId);
                                    }
                                    //回复的发布时间
                                    String replay_publish_time = "";
                                    Elements replyImgElements = null;
                                    JSONArray reply_img_urls = new JSONArray();
                                    if (type.equals("0")) {
                                        //发布时间
                                        replay_publish_time = replay_content.get("date") == null ? "" : replay_content.get("date").toString();
                                        //内容
                                        replay_data.put("content", FileUtils.filterImg(elements.get(i).select(".p_content").html()));
                                        replyImgElements = elements.get(i).select(".p_content").select("img.BDE_Image");
                                        if (replyImgElements.size() > 0) {
                                            for (int imgIndex = 0; imgIndex < replyImgElements.size(); imgIndex++) {
                                                org.jsoup.nodes.Element imgElement = replyImgElements.get(imgIndex);
                                                String img_url = imgElement.absUrl("src");
                                                reply_img_urls.add(img_url.replaceAll("\r|\n", ""));
                                            }
                                        }
                                    } else {
                                        //发布时间
                                        String tail_info = elements.get(0).select(".tail-info").text();
                                        replay_publish_time = getTime(tail_info);
                                        //内容
                                        replay_data.put("content", FileUtils.filterImg(replay_content.get("content") == null ? "" : replay_content.get("content").toString()));

                                        List<String> imgList = FileUtils.filterImgSrc(replay_content.get("content") == null ? "" : replay_content.get("content").toString());
                                        if (imgList != null && imgList.size() > 0) {
                                            for (String img : imgList) {
                                                reply_img_urls.add(img.replaceAll("\r|\n", ""));
                                            }
                                        }
                                    }
                                    //回复中的图片
                                    replay_data.put("img_urls", reply_img_urls);
                                    replay_data.put("publish_time", replay_publish_time);
                                    //回复的回复
                                    String post_id = replay_content.get("post_id").toString();
                                    BaiduForumDownloader downloader = this.cloneDownLoader();
                                    JSONArray replay_array = downloader.replayList(downloader, tid, post_id);
                                    replay_data.put("reply_nodes", replay_array);
                                    //回复人头像
                                    String replay_publish_user_photo = elements.get(i).select(".p_author_face ").select("img").attr("src");
                                    replay_data.put("publish_user_photo", replay_publish_user_photo.replaceAll("\r|\n", ""));
                                    //父级Id
                                    replay_data.put("parent_id", tid);
                                    //爬取时间
                                    replay_data.put("spider_time", sdf.format(new Date()));
                                    reply_nodes.add(replay_data);
                                }
                            } catch (Exception e) {
                                e.printStackTrace();
                                logger.error("获取本页回复有异常");
                                continue;
                            }
                        }
                        //获取后面几页的回复
                        JSONArray replys = new JSONArray();
                        List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
                        if (replayPageCount >= 2) {
                            List<FutureTask<JSONArray>> futureTasks = new ArrayList<FutureTask<JSONArray>>();
                            int threadNum = Integer.valueOf(PropertiesUtil.getVal("BaiduForum.replay.threanCount"));
                            ExecutorService executorService = Executors.newFixedThreadPool(threadNum);
                            ArrayList<ArrayList<Integer>> splitPages = CommonUtils.splitPages(2, replayPageCount, threadNum);
                            for (int i = 0; i < splitPages.size(); i++) {
                                BaiduForumDownloader page_downloader = this.cloneDownLoader();
                                ReplayCrawler replayCrawler = new ReplayCrawler(page_downloader, splitPages.get(i), url, tid, type, speechesId);
                                //FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(callable);
                                FutureTask<JSONArray> futureTask = new FutureTask<JSONArray>(replayCrawler);
                                futureTasks.add(futureTask);
                                executorService.submit(futureTask);
                            }
                            for (FutureTask<JSONArray> futureTask : futureTasks) {
                                JSONArray replyArrays = futureTask.get();
                                for (int i = 0; i < replyArrays.size(); i++) {
                                    JSONObject object = replyArrays.getJSONObject(i);
                                    replys.add(object);
                                }
                            }
                            if (replys.size() > 0 && replys != null) {
                                for (int i = 0; i < replys.size(); i++) {
                                    JSONObject reply_object = replys.getJSONObject(i);
                                    reply_nodes.add(reply_object);
                                }
                            }
                            if (reply_nodes.size() > 0 && reply_nodes != null) {
                                for (int i = 0; i < reply_nodes.size(); i++) {
                                    JSONObject reply_object = reply_nodes.getJSONObject(i);
                                    Map<String, Object> map = reply_object;
                                    list.add(map);
                                }
                            }
                            Collections.sort(list, new Comparator<Map<String, Object>>() {
                                public int compare(Map<String, Object> o1, Map<String, Object> o2) {
                                    String s1 = o1.get("publish_time").toString();
                                    String s2 = o2.get("publish_time").toString();
                                    int flag = s1.compareTo(s2);
                                    return flag;
                                }
                            });
                            executorService.shutdown();
                        } else {
                            if (reply_nodes.size() > 0 && reply_nodes != null) {
                                for (int i = 0; i < reply_nodes.size(); i++) {
                                    JSONObject reply_object = reply_nodes.getJSONObject(i);
                                    Map<String, Object> map = reply_object;
                                    list.add(map);
                                }
                            }
                        }
                        data.put("reply_nodes", reply_nodes);
                    }
                    item.put("data", data);
                    success = true;
                    break;
                }
            } catch (Exception e) {
                //System.out.println(url);
                //System.out.println(page.toString());
                Thread.sleep(1500);
                //e.printStackTrace();
                System.err.println("爬虫过程出现异常, 尝试重新爬取");
                continue;
            }
        }
        if(success == true) {
            return item;
        } else {
            throw new Exception(url+"爬取失败");
        }
    }

    /**
     * 每个帖子内容里面回复的回复
     *
     * @param downloader
     * @param tid
     * @param post_id
     * @return
     */
    public JSONArray replayList(BaiduForumDownloader downloader, String tid, String post_id) throws Exception {
        JSONArray array = new JSONArray();
        //默认取第一页
        Long tidLong = Long.parseLong(tid);
        Long post_idLong = Long.parseLong(post_id);
        int code = 0;
        String replay_url = "http://tieba.baidu.com/p/comment?tid=" + tidLong + "&pid=" + post_idLong + "&pn=1";
        try {
            Page pageContent = null;
            try {
                pageContent = downloader.getHtml(replay_url,null);
            } catch (Exception e) {
                e.printStackTrace();
                logger.error(e.getMessage(),e);
            }
            try {
                if (pageContent == null ||
                        (pageContent != null && pageContent.getHtml() == null) ||
                        (pageContent != null && pageContent.getStatusCode() == 403)) {
                    pageContent = downloader.getHtml(replay_url, true);
                }
            } catch (NullPointerException e) {
                pageContent = downloader.getHtml(replay_url, true);
                logger.error(e.getMessage(),e);
            }

            if (pageContent != null && pageContent.getHtml() != null) {
                org.jsoup.nodes.Document document = pageContent.getHtml().getDocument();
                //org.jsoup.nodes.Document document = downloader.get(replay_url,true);
                if (document != null) {
                    Elements first_elements = document.select(".lzl_single_post");
                    //第一页
                    if (first_elements != null) {
                        array = downloader.replayConetnt(first_elements, array);
                    }
                    //后面几页
                    org.jsoup.nodes.Element pageE = document.select(".lzl_li_pager").get(0);
                    if (pageE != null) {
                        JSONObject pageObj = JSONObject.parseObject(pageE.attr("data-field"));
                        int total = 1;
                        if (!pageE.attr("data-field").isEmpty()) {
                            total = pageObj.getInteger("total_page");
                        }
                        for (int index = 2; index < total; index++) {
                            String pageNum = index + "";
                            if (pageNum.matches("\\d") && !pageNum.equals("1")) {
                                String replay_page_url = "http://tieba.baidu.com/p/comment?tid=" + tidLong + "&pid=" + post_idLong + "&pn=" + pageNum;
                                //Elements elements = downloader.get(replay_page_url,true).select(".lzl_single_post");
                                Page replay = null;
                                replay = downloader.getHtml(replay_page_url, null);
                                if (replay == null && replay.getHtml() == null || replay.getStatusCode() == 403) {
                                    replay = downloader.getHtml(replay_page_url, true);
                                    if (replay == null || replay.getHtml() == null) {
                                        continue;
                                    } else {
                                        array = downloader.replayConetnt(replay.getHtml().getDocument().select(".lzl_single_post"), array);
                                    }
                                } else {
                                    System.err.println("获取后面几页的回复内容失败");
                                }
                            }
                        }
                    }
                } else {
                    System.err.println("获取回复内容失败");
                }
            }
        } catch (Exception e) {
            //e.printStackTrace();
            logger.error("回复的回复：" + replay_url, e);
        }
        return array;
    }

    private JSONArray replayConetnt(Elements elements, JSONArray array) {
        if (elements != null) {
            SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
            for (org.jsoup.nodes.Element element : elements) {
                try {
                    BaiduForumDownloader downloader =this.cloneDownLoader();
                    JSONObject data = new JSONObject();
                    JSONObject object = (JSONObject) JSONObject.parse(element.attr("data-field"));
                    String userName = object.getString("user_name");
                    String user_photo = element.select(".j_user_card.lzl_p_p").select("img").attr("src").replaceAll("\r|\n", "");
                    String content = element.select(".lzl_content_main").html();
                    String publish_time = element.select(".lzl_cnt").select(".lzl_content_reply").select(".lzl_time").text();
                    //获取用户id
                    String userId = downloader.queryUserId(userName);
                    data.put("publish_user", userName);
                    if(userId != null) {
                        data.put("publish_user_id", userId);
                    }
                    data.put("publish_user_photo", user_photo);
                    data.put("content", FileUtils.filterImg(content));
                    JSONArray img_urls = new JSONArray();
                    Elements imgElements = element.select(".lzl_content_main").select("img.BDE_Image");
                    for (int imgIndex = 0; imgIndex < imgElements.size(); imgIndex++) {
                        org.jsoup.nodes.Element imgElement = imgElements.get(imgIndex);
                        String img_url = imgElement.absUrl("src");
                        img_urls.add(img_url.replaceAll("\r|\n", ""));
                    }
                    data.put("img_urls", img_urls);
                    data.put("publish_time", publish_time);
                    data.put("spider_time", sdf.format(new Date()));
                    array.add(data);
                } catch (Exception e) {
                    //e.printStackTrace();
                    //logger.error(e.getMessage(),e);
                    continue;
                }
            }
        }
        return array;
    }


    /**
     * 回复爬取：爬取帖子后面页数的回复
     *
     * @author zh
     */
    public class ReplayCrawler implements Callable<JSONArray> {

        private BaiduForumDownloader downloader;
        private ArrayList<Integer> pages;
        private String url;
        private String parent_id;
        private String type;
        private String speeches_id;

        public ReplayCrawler(BaiduForumDownloader downloader, ArrayList<Integer> pages, String url, String parent_id, String type, String speeches_id) {
            // TODO Auto-generated constructor stub
            this.downloader = downloader;
            this.pages = pages;
            this.url = url;
            this.parent_id = parent_id;
            this.type = type;
            this.speeches_id = speeches_id;
        }

        @Override
        public JSONArray call() throws Exception {
            // TODO Auto-generated method stub
            JSONArray array = new JSONArray();
            for (int n = 0; n < pages.size(); n++) {
                try {
                    String replayUrl = url + "?pn=" + pages.get(n);
                    Page pageContent = null;
                    pageContent = getHtml(replayUrl, null);
                    if (pageContent == null || pageContent.getHtml() == null || pageContent.getStatusCode() == 403) {
                        pageContent = getHtml(replayUrl, true);
                    }
                    if (pageContent != null && pageContent.getHtml() != null) {

                        Elements elements = pageContent.getHtml().getDocument().select(".l_post");
                        File file = FileUtils.createTempFile(speeches_id + "_pageNo_" + pages.get(n), pageContent.getHtml().getDocument().html().toString());
                        files.put(speeches_id + "_pageNo_" + pages.get(n), file);

                        for (int m = 0; m < elements.size(); m++) {
                            JSONObject replay_data = new JSONObject();
                            JSONObject replay_floor = JSONObject.parseObject(elements.get(m).attr("data-field"));
                            if (replay_floor != null) {
                                JSONObject replay_content = (JSONObject) JSONObject.parse(replay_floor.get("content").toString());
                                if (replay_floor == null) {
                                    continue;
                                } else {
                                    //回复人
                                    JSONObject replay_author = JSONObject.parseObject(replay_floor.get("author").toString());
                                    replay_data.put("publish_user", replay_author.get("user_name"));
                                    //回复的发布时间
                                    String replay_publish_time = "";
                                    Elements replyImgElements = null;
                                    JSONArray reply_img_urls = new JSONArray();
                                    if (type.equals("0")) {
                                        //发布时间
                                        replay_publish_time = replay_content.get("date") == null ? "" : replay_content.get("date").toString();
                                        //内容
                                        replay_data.put("content", FileUtils.filterImg(elements.get(m).select(".p_content").html()));
                                        replyImgElements = elements.get(m).select(".p_content").select("img.BDE_Image");
                                        if (replyImgElements.size() > 0) {
                                            for (int imgIndex = 0; imgIndex < replyImgElements.size(); imgIndex++) {
                                                org.jsoup.nodes.Element imgElement = replyImgElements.get(imgIndex);
                                                String img_url = imgElement.absUrl("src");
                                                reply_img_urls.add(img_url.replaceAll("\r|\n", ""));
                                            }
                                        }
                                    } else {
                                        //发布时间
                                        String tail_info = elements.get(0).select(".tail-info").text();
                                        replay_publish_time = getTime(tail_info);
                                        //内容
                                        replay_data.put("content", FileUtils.filterImg(replay_content.get("content") == null ? "" : replay_content.get("content").toString()));
                                        List<String> imgList = FileUtils.filterImgSrc(replay_content.get("content") == null ? "" : replay_content.get("content").toString());
                                        if (imgList != null && imgList.size() > 0) {
                                            for (String img : imgList) {
                                                reply_img_urls.add(img.replaceAll("\r|\n", ""));
                                            }
                                        }
                                    }
                                    replay_data.put("img_urls", reply_img_urls);
                                    replay_data.put("publish_time", replay_publish_time);
                                    //回复的回复
                                    String post_id = replay_content.get("post_id").toString();
                                    BaiduForumDownloader downloader1 = downloader.cloneDownLoader();
                                    JSONArray replay_array = replayList(downloader1, parent_id, post_id);
                                    replay_data.put("reply_nodes", replay_array);
                                    //回复人头像
                                    String replay_publish_user_photo = elements.get(m).select(".p_author_face ").select("img").attr("src");
                                    replay_data.put("publish_user_photo", replay_publish_user_photo.replaceAll("\r|\n", ""));
                                    //父级Id
                                    replay_data.put("parent_id", parent_id);
                                    //爬取时间
                                    //爬取时间
                                    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                                    replay_data.put("spider_time", sdf.format(new Date()));
                                    array.add(replay_data);
                                }
                            }

                        }
                    }
                } catch (Exception e) {
                    // TODO: handle exception
                    logger.error(e.getMessage(),e);
                    e.printStackTrace();
                    continue;
                }

            }
            return array;
        }

    }


    public BaiduForumDownloader cloneDownLoader() throws InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException {
        BaiduForumDownloader loader = new BaiduForumDownloader();
        loader.cookies = (HashSet<Cookie>) this.cookies.clone();
        loader.headers = (HashMap<String, String>) this.headers.clone();
        return loader;
    }

    /**
     *获取userId
     * @param name
     * @return
     * @throws Exception
     */
    public String queryUserId(String name) {
        String url = "http://tieba.baidu.com/home/get/panel?un=" + name;
        //System.out.println("url:   " + url);
        String queryUserId = null;
        boolean success = false;
        long startTime = System.currentTimeMillis();
        long nowTime = startTime;
        while (nowTime - startTime < 30000L) {
            nowTime = System.currentTimeMillis();
            Page pageContent = null;
            try {
                pageContent = getHtml(url, true);
                if(null != pageContent) {
                    if(pageContent.getStatusCode() == 403 && null == pageContent.getHtml()){
                        continue;
                    }
                    String json = pageContent.getRawText();
                    if(json == "null" || null == json || "" == json) {
                        continue;
                    }
                    //System.out.println(json);
                    JSONObject object = JSONObject.parseObject(json);
                    queryUserId = (object.getJSONObject("data")).getString("id");
                }
            } catch (Exception e) {
                //e.printStackTrace();
                try {
                    Thread.sleep(3000);
                } catch (InterruptedException e1) {
                    e1.printStackTrace();
                }
                continue;

            }

        }
        if(success == true) {
            return queryUserId;
        } else {
            return null;
            //throw new Exception("queryUserId:  " + url + "  爬取失败");
        }
    }

}
